package com.qen.zhihulanzhou

import cn.hutool.core.io.FileUtil
import cn.hutool.core.io.StreamProgress
import cn.hutool.core.io.file.FileWriter
import cn.hutool.core.lang.Console
import cn.hutool.core.util.StrUtil
import cn.hutool.http.HttpUtil
import cn.hutool.json.JSONUtil
import cn.hutool.poi.excel.ExcelUtil
import io.github.bonigarcia.wdm.WebDriverManager
import it.skrape.core.htmlDocument
import it.skrape.fetcher.BrowserFetcher
import it.skrape.fetcher.Method
import it.skrape.fetcher.response
import it.skrape.fetcher.skrape
import okhttp3.OkHttpClient
import okhttp3.Request
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import java.time.Duration
import java.util.regex.Pattern

const val DownloadPath = "D:\\codework\\JavaWorks\\kotlin-skrape\\out\\lanzhou\\download"
const val BaseUrl = "https://wwa.lanzoux.com"
fun main() {
//    System.setProperty("webdriver.chrome.driver", "/Users/a1/Documents/Git/Kotlin/Eleven/chrome/chromedriver")
    WebDriverManager.chromedriver().setup()
    println("进来了")

    val reader = ExcelUtil.getReader("$LanZhouExcelPath/All.xlsx")
//    val list = reader.read(2)
    val list = reader.read(511)
    val driver: WebDriver = ChromeDriver()
    val errorUrls = ArrayList<String>()
    val writer: FileWriter = FileWriter("errorUrl.txt")
    try {
        for (item in list) {
            val id = item.get(0).toString()
            driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(30))
            driver.get("${BaseUrl}/${id}")
            val title = driver.title
            println(title)

            val wait = WebDriverWait(driver, Duration.ofSeconds(30))
            val iframe = wait.until(ExpectedConditions.presenceOfElementLocated(By.className("ifr2")))
            driver.switchTo().frame(iframe)
            val element: WebElement = wait.until(ExpectedConditions.presenceOfElementLocated(By.id("tourl")))
            if (element == null) {
                println("没有发现元素:$title")
            } else {
                val a = element.findElement(By.tagName("a"))
                val href = a.getAttribute("href") ?: ""
                val fileUrl = getLocation(href)
                val str = downloadFile(fileUrl, item.get(1).toString())
                if(StrUtil.isNotBlank(str)){
                    writer.append("$str\n")
                }
//                downloadFile(fileUrl, getTitle(title))
//            val elementList = element.findElements(By.className("txt"))
//            val elementBtn = elementList.get(2)
//            elementBtn.click()
            }
//    val element: WebElement = driver.findElement(By.className("load"))
        }
    }catch (e:Exception){
        e.printStackTrace()
    } finally {
        driver.quit()
    }
    println("结束了")
    Console.log(errorUrls)

}

fun skrapeGo() {
    val id = "iA47bqyj67c"
    skrape(BrowserFetcher) {
        request {
            url = "${BaseUrl}/${id}"
            userAgent = UserAgent
            method = Method.GET
        }
        response {
            htmlDocument {
                ".ifr2" {
                    findFirst {
                        Console.log(this)
                        val src = this.attribute("src")
                        goSrc(src)
                    }
                }
            }
        }
    }
}

fun goSrc(src: String) {
    skrape(BrowserFetcher) {
        request {
            url = "${BaseUrl}${src}"
            userAgent = UserAgent
            method = Method.GET
        }
        response {
//            Console.log("外部:${this.responseStatus}")
//            Console.log("外部:${this.responseBody}")
            val html = this.responseBody
            val wsk_signPattern = Pattern.compile("var\\s+wsk_sign\\s*=\\s*'([^']*)';")
            val wsk_sign = getValue(wsk_signPattern, html)

            val aihidcmsPattern = Pattern.compile("var\\s+aihidcms\\s*=\\s*'([^']*)';")
            val aihidcms = getValue(aihidcmsPattern, html)

            val ciucjdsdcPattern = Pattern.compile("var\\s+ciucjdsdc\\s*=\\s*'([^']*)';")
            val ciucjdsdc = getValue(ciucjdsdcPattern, html)

            val ws_signPattern = Pattern.compile("var\\s+ws_sign\\s*=\\s*'([^']*)';")
            val ws_sign = getValue(ws_signPattern, html)

            val sasignPattern = Pattern.compile("var\\s+sasign\\s*=\\s*'([^']*)';")
            val sasign = getValue(sasignPattern, html)

            val ajaxdataPattern = Pattern.compile("var\\s+ajaxdata\\s*=\\s*'([^']*)';")
            val ajaxdata = getValue(ajaxdataPattern, html)

            val kdns = 1

            val urlPattern = Pattern.compile("url\\s:\\s'([^']*)',")
            val goUrl = getValue(urlPattern, html)

            val signPattern =
                Pattern.compile("data\\s:\\s\\{\\s'action':'downprocess','signs':ajaxdata,'sign':'([^']*)','websign':ciucjdsdc,'websignkey':aihidcms,'ves':1,'kd':kdns\\s},")
            val sign = getValue(signPattern, html)
            Console.log(
                """
                {
                            "action" to "downprocess"
                            "signs" to $ajaxdata
                            "sign" to $sign
                            "websign" to $ciucjdsdc
                            "websignkey" to $aihidcms
                            "ves" to 1
                            "kd" to $kdns
                        }
            """.trimIndent()
            )
            val ajaxdataCode = URLEncoder.encode(ajaxdata, StandardCharsets.UTF_8.toString())
            Console.log("Ajax.PHP地址：${BaseUrl}${goUrl}")
            Console.log("action=downprocess&signs=$ajaxdata&sign=$sign&websign=$ciucjdsdc&websignkey=$aihidcms&ves=1&kd=$kdns")
            //----下载链接
            skrape(BrowserFetcher) {
                request {
                    url = "${BaseUrl}${goUrl}"
                    userAgent = UserAgent
                    method = Method.POST
                    headers = ajaxHeader()
                    body {
                        form {
                            "action" to "downprocess"
                            "signs" to "${ajaxdata}"
                            "sign" to "${sign}"
                            "websign" to "${ciucjdsdc}"
                            "websignkey" to "${aihidcms}"
                            "ves" to 1
                            "kd" to "${kdns}"
                        }
                    }
//                    body = "action=downprocess&signs=$ajaxdataCode&sign=$sign&websign=$ciucjdsdc&websignkey=$aihidcms&ves=1&kd=$kdns"
                }
                response {
//                    Console.log("内部：${this.responseStatus}")
//                    Console.log("内部：${this.responseBody}")
                    if (this.responseStatus.code == 200) {
                        val json = JSONUtil.parseObj(this.responseBody)
                        val dom = json.getStr("dom")
                        val url = json.getStr("url")
                        val downloadUrl = "${dom}/file/${url}"
                        Console.log(downloadUrl)
//                        HttpUtil.downloadFile(downloadUrl, FileUtil.file(DownloadPath))
                        val fileUrl = getLocation(downloadUrl)
                        downloadFile(fileUrl, "")
                    }
                }
            }
            // -----下载链接

        }
    }
}

fun getValue(pattern: Pattern, html: String): String {
    val matcher = pattern.matcher(html)
    if (matcher.find()) {
        val wskSignValue = matcher.group(1)
        return wskSignValue
    } else {
        return ""
    }
}


//https://down-load.lanrar.com/file/?VzECPA8+Dj9RWAM7BDFVOVdoATkAuFfKV/VSsFSxBokG4QLOAM0GuwXTA9wHc1wCUykO4lKKAZIB4lPtUodbslfwAscPtA71UZUDsATDVdVX6gGKANRXsFfLUulU7wadBoICsADmBugFtAP+B9hctVOJDo5S4gG/AYpTsFKLW+xXsgL7D8wOvFGfA/oEtFXsV40B4gDLV/pXtFLsVKUG4gadAugAtAbKBcwDsQfvXNBTJw57UmkBZQEgUzNSZ1s2VzICCg87Dj5ROgNhBGdVZFc9ATwAaFdiV2JSclRuBnQGOwJgAGkGZAVnA2IHZ1xoUy8Oe1J9AT4BNFNlUjxba1d4AmUPZQ51UTcDZgR/VWFXaQE8AGxXZVdkUjVUaQYxBj8CNAA+BjEFawM3B2BcOFNsDjtSOQFnAWNTbFJqW2tXNAJtDzMOOFFiAzYEYlV6V30BYAA5V3NXJVInVG0GdQZvAjEAYAZjBWMDZgdiXG9TOw45UisBdwFvUzhSa1s8V2oCZA9lDmlRNwNkBGBVZVc5ATYAaFd7V35SclRuBmsGcQJoAGwGZQVhA2wHZFxpUzgOOVI4ATQBIFMgUn5bLVdqAmQPZA5oUTIDZwRpVWZXNAE1AGZXc1clUj1UeAY6BjQCZQBvBnwFagNiB31cbFM8DjNSIwEyATNTYVIgW3xXMwI6DyUONFFYAzYEO1VpVz0=

fun main2() {
    val html = """
        var lanosso = '';
				if (typeof(killdns2)=='undefined')
		{
			var lanosso = '&lanosso';
		}
				var wsk_sign = 'c20230908';
		var aihidcms = 'InA6';
		var ciucjdsdc = '';
		var ws_sign = 'c20230818';
		var sasign = 't';
		var ajaxdata = '?ctdf';
		var kdns =1;
		if (typeof(killdns)=='undefined'){
					var kdns = 0;
		}
		${'$'}.ajax({
			type : 'post',
			url : '/ajaxm.php?file=47008785',
			data : { 'action':'downprocess','signs':ajaxdata,'sign':'AGZTbV5vU2IIAVZpUWFSbls1V2VWOQs5BTpWZwFlBjJSdFJxDm5QNVU0VjMDZwU0UzBUZAJlBzEGNA_c_c','websign':ciucjdsdc,'websignkey':aihidcms,'ves':1,'kd':kdns },
			dataType : 'json',
			success:function(msg){
				var date = msg;
				var dom_down = date.dom;
				if(date.zt == '1'){
										${'$'}("#tourl").html("<a href="+dom_down+"/file/"+ date.url + lanosso +" target=_blank rel=noreferrer><span class=txt>电信下载</span><span class='txt txtc'>联通下载</span><span class=txt>普通下载</span></a>");
					setTimeout('${'$'}("#outime").css("display","block");',1800000);
									}else{
					${'$'}("#tourl").html("网页超时，请刷新");
				};
				
			},
			error:function(){
				${'$'}("#tourl").html("获取失败，请刷新");
			}
	
		});
    """.trimIndent()
    val wsk_signPattern = Pattern.compile("var\\s+wsk_sign\\s*=\\s*'([^']*)';")
    val wsk_sign = getValue(wsk_signPattern, html)
    Console.log(wsk_sign)
    val aihidcmsPattern = Pattern.compile("var\\s+aihidcms\\s*=\\s*'([^']*)';")
    val aihidcms = getValue(aihidcmsPattern, html)
    Console.log(aihidcms)
    val ciucjdsdcPattern = Pattern.compile("var\\s+ciucjdsdc\\s*=\\s*'([^']*)';")
    val ciucjdsdc = getValue(ciucjdsdcPattern, html)
    Console.log(ciucjdsdc)
    val ws_signPattern = Pattern.compile("var\\s+ws_sign\\s*=\\s*'([^']*)';")
    val ws_sign = getValue(ws_signPattern, html)
    Console.log(ws_sign)
    val sasignPattern = Pattern.compile("var\\s+sasign\\s*=\\s*'([^']*)';")
    val sasign = getValue(sasignPattern, html)
    Console.log(sasign)
    val ajaxdataPattern = Pattern.compile("var\\s+ajaxdata\\s*=\\s*'([^']*)';")
    val ajaxdata = getValue(ajaxdataPattern, html)
    Console.log(ajaxdata)
    val kdns = 1
    Console.log(kdns)
    val urlPattern = Pattern.compile("url\\s:\\s'([^']*)',")
    val goUrl = getValue(urlPattern, html)
    Console.log(goUrl)
    val signPattern =
        Pattern.compile("data\\s:\\s\\{\\s'action':'downprocess','signs':ajaxdata,'sign':'([^']*)','websign':ciucjdsdc,'websignkey':aihidcms,'ves':1,'kd':kdns\\s},")
    val sign = getValue(signPattern, html)
    Console.log(sign)
}
//yield scrapy.Request(url, callback=self.parse, meta={'dont_redirect': True, 'handle_httpstatus_list': [302]})

fun main1() {
    val goUrl =
        "https://down-load.lanrar.com/file/?UDZaZFtqBzYFDAY+UWRQPFRrAzte5lbLBKYE5gfiVtkJ7gXJXJEEuVKEV4gBdQFfBH4F6VKKVsUK6VPtVIFbslD3Wp9b4Af8BcEGtVGWUNBU6QOIXopWsQSYBL8HvFbNCY0Ft1y6BOpS41eqAd4B6ATeBYVS4lboCoFTsFSNW+xQtVqjW5gHtQXLBv9R4VDpVI4D4F6VVvsE5wS6B/ZWsgmSBe9c6ATIUptX5QHpAY0EcAVwUmlWMgorUzNUYVs2UDVaUltvBzcFbgZkUTJQYVQ+Az5eNlZjBDEEJAc9ViQJNAVnXDUEZlIwVzYBYQE1BHgFcFJ9VmkKP1NlVDpba1B/Wj1bMQd8BWMGY1EqUGRUagM+XjJWZAQ3BGMHOlZhCTAFM1xiBDNSPFdjAWYBZQQ7BTBSOVYwCmhTbFRsW2tQM1o1W2cHMQU2BjNRN1B/VH4DYl5nVnIEdgRxBz5WJQlgBTZcPARhUjRXMgFkATIEbAUyUitWIApkUzhUbVs8UG1aPFsxB2AFYwZhUTNQZVQ9AzZeMlZ6BC0EJAc9VjsJfgVvXDAEZ1I3VzEBZAE2BGwFNVI6VmMKK1MgVHhbLVBtWjxbMAdhBWYGYlE8UGNUNwM3XjhWcgR2BGsHK1ZqCTsFYlwzBH5SPVc2AXsBMQRrBThSI1ZlCjhTYVQmW3xQNFpiW3EHPQUMBjNRblBsVD4="
    val client = OkHttpClient.Builder().followRedirects(false).build()
    val request = Request.Builder().url(goUrl)
        .addHeader(
            "accept",
            "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
        )
        .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
        .addHeader("cache-control", "no-cache")
        .addHeader("pragma", "no-cache")
        .addHeader("priority", "u=0, i")
        .addHeader("sec-ch-ua", "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"")
        .addHeader("sec-ch-ua-mobile", "?0")
        .addHeader("sec-ch-ua-platform", "\"macOS\"")
        .addHeader("sec-fetch-dest", "document")
        .addHeader("sec-fetch-mode", "navigate")
        .addHeader("sec-fetch-site", "none")
        .addHeader("sec-fetch-user", "?1")
        .addHeader("upgrade-insecure-requests", "1")
        .build()
    val response = client.newCall(request).execute()
    Console.log(response)
    val status = response.code
    val location = response.headers("Location")
    Console.log(location)
    Console.log(response.headers)
//    val goUrl = "https://c1029.lanosso.com/3c4453455aaaf242eb41b1558b227804/67203699/2021/07/03/5d8304aa69acc8b3ee04de9d1c8abec2.pdf?fn=%E7%9F%A5%E4%B9%8E%E7%9B%90%E9%80%89%20_%20%E9%87%91%E4%B8%89%E8%A7%92%E6%AF%92%E5%93%81%E7%8E%8B%E5%9B%BD%E7%9A%84%E5%BB%BA%E7%AB%8B%E8%80%85%EF%BC%8C%E5%85%B6%E5%AE%9E%E6%98%AF%E4%B8%80%E6%94%AF%E4%B8%AD%E5%9B%BD%E9%98%9F%E4%BC%8D.pdf"
//    HttpUtil.downloadFileFromUrl(goUrl, FileUtil.file("${DownloadPath}/test.pdf"),60000,object: StreamProgress{
//        override fun start() {
//            Console.log("Start")
//        }
//
//        override fun progress(total: Long, progressSize: Long) {
//            Console.log("total:$total;progressSize:$progressSize")
//        }
//
//        override fun finish() {
//            Console.log("finish")
//        }
//    })
//    val writer = FileWriter("${DownloadPath}/test.pdf");
//    skrape(BrowserFetcher) {
//        request {
//            url = goUrl
//            userAgent = UserAgent
//            method = Method.GET
//            headers = goHeader()
//            cookies = goCookie()
//            followRedirects = true
//        }
//        response {
//            Console.log(this.headers)
//            val steam = this.responseBody.byteInputStream()
//            writer.writeFromStream(steam)
//            Console.log("responseStatus：${this.responseStatus}")
//        }
//    }

//    val res = HttpRequest.get(goUrl)
//        .setFollowRedirects(true)
//        .timeout(60000)
//        .execute()
//    Console.log(res.status)
//    res.writeBodyForFile(File(DownloadPath + "/test.pdf"), object : StreamProgress {
//        override fun start() {
//            Console.log("++++++++++start===========")
//        }
//        override fun progress(total: Long, progressSize: Long) {
//            Console.log("total:$total,progressSize:$progressSize")
//        }
//        override fun finish() {
//            Console.log("++++++++++finish===========")
//        }
//    })

//    val req = HttpUtil.createGet(goUrl,false)
//    val res = req.execute()
//    Console.log(res)
//    val location = res.header("Location")
//    Console.log(location)
//    println(res.status)
//    println(res.body())

//    Console.log(getRedirectUrl(goUrl))
}

fun goHeader(): MutableMap<String, String> {
    return mutableMapOf(
        "accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "accept-language" to "zh-CN,zh;q=0.9,en;q=0.8",
        "cache-control" to "no-cache",
        "pragma" to "no-cache",
        "priority" to "u=0, i",
        "sec-ch-ua" to "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
        "sec-ch-ua-mobile" to "?0",
        "sec-ch-ua-platform" to "\"macOS\"",
        "sec-fetch-dest" to "document",
        "sec-fetch-mode" to "navigate",
        "sec-fetch-site" to "none",
        "sec-fetch-user" to "?1",
        "upgrade-insecure-requests" to "1"
    )
}

fun goCookie(): MutableMap<String, String> {
    return mutableMapOf(
        "down_ip" to "1"
    )
}

/**
 * 处理跳转链接，获取重定向地址
 * @param url   源地址
 * @return      目标网页的绝对地址
 */
//fun getAbsUrl(url: String): String {
//    val httpclient = HttpClients.createDefault()
//    val context: HttpClientContext = HttpClientContext.create()
//    val httpget = HttpGet(url)
//    var response: CloseableHttpResponse? = null
//    var absUrl = ""
//    try {
//        response = httpclient.execute(httpget, context)
//        val target: HttpHost = context.getTargetHost()
//        val redirectLocations = context.getRedirectLocations()
//        val location = URIUtils.resolve(httpget.getURI(), target, redirectLocations);
//        absUrl = location.toASCIIString();
//    } catch (e: Exception) {
//        e.printStackTrace();
//    } finally {
//        try {
//            httpclient.close();
//            response?.close();
//        } catch (e: Exception) {
//            e.printStackTrace();
//        }
//    }
//    return absUrl;
//}

/**
 * 获取重定向地址
 * @param path 原地址
 * @return
 * @throws Exception
 */
//fun getRedirectUrl(path: String): String {
//    val conn: URLConnection = URL(path).openConnection();
////    conn.setInstanceFollowRedirects(false);
//    conn.allowUserInteraction = true
//    conn.setConnectTimeout(5000);
//    return conn.getHeaderField("Location");
//}

fun getLocation(goUrl: String): String {
    val client = OkHttpClient.Builder().followRedirects(false).build()
    val request = Request.Builder().url(goUrl)
        .addHeader(
            "accept",
            "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
        )
        .addHeader("accept-language", "zh-CN,zh;q=0.9,en;q=0.8")
        .addHeader("cache-control", "no-cache")
        .addHeader("pragma", "no-cache")
        .addHeader("priority", "u=0, i")
        .addHeader("sec-ch-ua", "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"")
        .addHeader("sec-ch-ua-mobile", "?0")
        .addHeader("sec-ch-ua-platform", "\"macOS\"")
        .addHeader("sec-fetch-dest", "document")
        .addHeader("sec-fetch-mode", "navigate")
        .addHeader("sec-fetch-site", "none")
        .addHeader("sec-fetch-user", "?1")
        .addHeader("upgrade-insecure-requests", "1")
        .build()
    val response = client.newCall(request).execute()
//    Console.log(response)
    val status = response.code
    val location = response.headers("Location")
    Console.log(location)
//    Console.log(response.headers)
    return location.get(0)
}

fun downloadFile(gourl: String, fileName: String):String {
    var str = gourl
    try{
        HttpUtil.downloadFileFromUrl(gourl, FileUtil.file("${DownloadPath}/${fileName}"))
//        HttpUtil.downloadFileFromUrl(gourl, FileUtil.file("${DownloadPath}/${fileName}"), 60000, object : StreamProgress {
//            override fun start() {
//                Console.log("Start")
//            }
//            override fun progress(total: Long, progressSize: Long) {
//                Console.log("total:$total;progressSize:$progressSize")
//            }
//            override fun finish() {
//                HttpUtil.closeCookie()
//                Console.log("finish:${DownloadPath}/${fileName}")
//            }
//        })
        return ""
    }catch (e:Exception){
        e.printStackTrace()
        return str
    }
}

fun ajaxHeader(): MutableMap<String, String> {
    return mutableMapOf(
        "accept" to "application/json, text/javascript, */*",
        "accept-language" to "zh-CN,zh;q=0.9,en;q=0.8",
        "cache-control" to "no-cache",
        "content-type" to "application/x-www-form-urlencoded",
        "pragma" to "no-cache",
        "priority" to "u=1, i",
        "sec-ch-ua" to "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
        "sec-ch-ua-mobile" to "?0",
        "sec-ch-ua-platform" to "\"macOS\"",
        "sec-fetch-dest" to "empty",
        "sec-fetch-mode" to "cors",
        "sec-fetch-site" to "same-origin",
        "x-requested-with" to "XMLHttpRequest"
    )
}

fun mainx() {
    val str = HttpUtil.post(
        "https://wwa.lanzoux.com/ajaxm.php?file=47008772",
        "action=downprocess&signs=%3Fctdf&sign=VTMCPAEwBjdWX1ZpVGRWalE_aAjBQPwc1BzgHNgZtVGcILgckD29SNwFgBmMEYFFhVjoAM1I_bAzICNg_c_c&websign=&websignkey=BaTy&ves=1&kd=1"
    )
    Console.log(str)
}


fun getTitle(inTitle: String): String {
    if (inTitle.contains(".pdf")) {
        val s = inTitle.substring(0, inTitle.lastIndexOf(".pdf") + 4)
        return s
    } else {
        return inTitle + ".pdf"
    }
}

fun main0() {
    val reader = ExcelUtil.getReader("$LanZhouExcelPath/All.xlsx")
    val list = reader.read(2)
    Console.log(list.joinToString("\n"))
}